{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import random, choice\n",
    "from agents import SarsaAgent, QAgent\n",
    "from gridworld import CliffWalk,CliffWalk2\n",
    "from gym import Env\n",
    "import gym\n",
    "\n",
    "from utils import greedy_policy, learning_curve, str_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = CliffWalk()\n",
    "env.reset()\n",
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10000/10000 [00:04<00:00, 2354.73it/s]\n",
      "100%|██████████| 10000/10000 [00:04<00:00, 2464.19it/s]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "q_agent = QAgent(env, capacity = 10000)\n",
    "sarsa_agent = SarsaAgent(env, capacity = 10000)\n",
    "\n",
    "sarsa_sta = sarsa_agent.learning(display = False,\n",
    "                                 max_episode_num = 10000, \n",
    "                                 epsilon = 0.1, \n",
    "                                 decaying_epsilon = False)\n",
    "q_sta = q_agent.learning(display = False,\n",
    "                         max_episode_num = 10000,\n",
    "                         epsilon = 0.1, \n",
    "                         decaying_epsilon = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEWCAYAAACnlKo3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzsnXlYXdW5/z8vB3IgAZKQCCQcIkRDFKJEQJGIGWoc61C1Dmm1Dq29uWpnbfU6RWtuvdbf7WSr3taprVXbeus8VHslkTZGAwYToiExREM0RCEmkAAJsH5/7A05hzNwOMM+w16f59nPOXuttdd6v3t6915r7bVEKYVGo9FoNKGQEmsDNBqNRpO4aCei0Wg0mpDRTkSj0Wg0IaOdiEaj0WhCRjsRjUaj0YSMdiIajUajCRntRDSaKCEieSKyUkS6ROT/xdoeX4iIEpHDY22HJnHRTkSjiR7fBD4DspVSP/CVQETmicj/mY5mt4g8KyJHWGumb0SkTET+LiKdIvK5iDSIyBmxtksTX2gnotGYiEhqhLM8FNig/HzRKyI1wN+BZ4DpQDHwLvBPESmKsC2h8BzwKpAP5ALfBvaMNRMx0PeaZEUppRe9RGUBCoH/BT4FOoB7zfAU4GbgQ2An8HtgohlXBCjgCmAbsAtYChyLcYP9fCgfM/3lwD+Be4HdwPvASW7xVwDvAV3AFuDf3OIWAm3Aj4AdwB/M8DOBtWZZ/wKODqBxHvC2WfbbwDwz/BHgALAf6AYW+9j2DeA3PsJfAh72U95hwP+Z+/Mz4DFgklv8VuA6c1/tBp4E0t3irwc+AT4GrjT39eE+yplqxk3yY8dk4Hnz2O4y/7vc4uuA5eax6QEON4/VFvNYtAJfDUaTXuJ7ibkBeknOBXAATcDPgAlAOlBrxl0JbAZmApkYjmboBj7kRO43tzkF6AWexngaLsBwPAvM9JcD/cD3gDTgIvPmmWPGf9G8SQmwANgHVJhxC81t/wtwAhnAMWb+1aaGy8wbs9OHxhzzBnopkAosMdenmPGPAHf62T/jgQFgkY+4K4DtfrY7HDjZtPcQYCXwc7f4rcBbGG82ORgOdKkZdxrQDswxj8mf8O9EBNhkOocvAXkj4qcA55s6soC/AE+7xdcBHwFl5r6ZiPEWM9uMnwaUBaNJL/G9xNwAvSTnAtRgPKWm+oj7B3C12/psjKf2VDcnUuAW3wFc5Lb+FPBd8//lGE/V4hb/FnCpH7ueBr5j/l+I8abg/qR+H/DjEdtsxHRaI8IvBd4aEbYKuNz8H8iJuEydR/iIOw3YH+R+/hLwjtv6VuASt/W7gfvN/w8Bd7nFlfhzIm423gt8AAyaN/dZftLOBXa5rdcBd7itT8B4szsfyBiLJr3E96LrKTXRohD4UCnV7yNuOkZV1hAfYjiQPLewdrf/PT7WM93Wtyvz7uOW33QAETldRN4cahwGzsCoqhniU6VUr9v6ocAPzIbkz81tCofyG0XHUNkFPtKOZBfGjXmaj7hpGNU6Xpg9vp4Qke0isgf44wg9YFTNDbGPg/tqOkYVobutflFKtSmlrlVKHYaxX/ZiVD0iIuNF5AER+dC0YyUwSUQcbllsc8trL8Zb4lLgExF5YagDQZCaNHGKdiKaaLENmOGnsfpjjJvSEDMwqpXafaQNhgIRkRH5fSwiToy3lnswqmMmAS9iVNUMMbLRexuwXCk1yW0Zr5R6PAgdQ2VvH81g86a6CrjAR/SFGE/yvvhP0+ajlFLZwCV46gnEJxgO0d3WoFBKbQN+jVEVBvADjDfIatOO+Wa4332rlHpFKXUyhpN8H/itGRWOJk2M0U5EEy3ewrhp3SUiE0QkXUROMOMeB74nIsUikolxE3nSz1tLMOQC3xaRNBG5ADgSw1mMw6hn/xToF5HTMdpYAvFbYKmIVJu9iiaIyBdFJMtH2heBEhH5ioikishFQClGO0Iw3ABcJiLfFpEsEZksIncCJ2LsE19kYTTU7xaRAoyG8mD5M3C5iJSKyHjgNn8JTVtuF5HDRSRFRKZitGW96WZHD/C5iOQEysvML09EzhGRCUCfqWEwApo0MUY7EU1UUEoNAGdhNJp+hNEL6iIz+iHgDxhVIK0YDeffCqO41cAsjCqg5cCXlVIdSqkujG6pf8aoPvoK8Owodq8BrsJoC9iF0QHgcj9pOzB6cv0Ao93mh8CZSimfVVE+tq8HTgXOw3C4nRgN+Scppdb72ex2oAKj88ALGJ0SgkIp9RLwc4yeUJvNX3/sx2ifeg2jQXw9xs3/cjP+5xgdET7DcCwvj1J8CvB9jLe3ToxODv8eriZN7BHPqmSNJrEQkcuBbyilamNtS7iIyNHA68BXlFKvxNoejSYY9JuIRhMnKKXexeiZdFQUPnzUaKKCPlE1mjhCKfUGxkeIGk1CoKuzNBqNRhMyCVedJSKnichGEdksIjfE2h6NRqOxMwn1JmJ+yNSCMURCG8ZYRUuUUhv8bTN16lRVVFRkjYEajUaTBDQ0NHymlDokmLSJ1iZyHLBZKbUFQESeAM4B/DqRoqIi1qxZE1Jh77//PkccERejcluG3TTbTS9ozXYhHM0iEnA0A3cSrTqrAM9hG9oIboiJkNixY8foiZIMu2m2m17Qmu2CVZoT7U0kKERkGeYXtLm5udTV1XnE5+fnU1xcTEtLC2VlZaxcudIrj5qaGnp7e+nu7qa9vZ1t27Z5xBcUFOByuWhtbaWkpIT6+nqvPGpra2lpaaG4uJi2tja2b/ccDaOwsJC8vDza2tooLi5m1apVXnnMnz+f5uZmSkpKaG1t9ToxioqKyMnJob29HZfLxerVq0fuCxYsWEBTUxNlZWW0tLSwc+dOjzQzZ84kKyuLzs5OBgYGvPZXamoqtbW1NDY2MnfuXJqbm+no6PBIM2vWLJxOJ93d3eTk5NDY2OgR73Q6qampoaGhgcrKSpqamti1a5dHmtmzZ+NwOOjt7SUrK4umpiaP+IyMDKqrq4fzaGxsZM8ez+ktSktLGRgYYHBwEKfTyfr1nt/sZWZmUlVVNZzHvn37vPTOmTOHvr4+UlJScDgcbNjg+aKbnZ1NRUXFcB6rV6+mp6fHI015eTldXV2kp6czMDDAxo0bPeInT55MeXn5cB6rVq2ir6/PI01FRQWdnZ1kZmbS19fHpk2bPOKnTJlCWVkZa9eupaKigvr6evr7PT/6r6qqor29nZycHLq6utiyZQvd3d3DmnNzcykpKaG5uZny8nJWrFjByCru6upq2trayMvLo7Ozk61bt3rEB3s9tba24nK5YnI9DWm2+nrKy8vzqgmx6no6cOAAO3fuDOt6CoZEaxOpAZYppU41128EUEr9xN82VVVVKtTqrLq6OhYuXBjStomK3TTbTS9ozXYhHM0i0qCUqgombaJVZ70NzDLHXBoHXMwow1hoNBqNJnokVHWWUqpfRK4FXsGYMOghpVRztMqzY68uu2m2m17Qmu2CVZoTyokAKKVexBg9Nerk5ORYUUxcYTfNdtMLWrNdsEpzolVnWUp7e6jTWyQudtNsN72gNdsFqzQn3JuIlbhcLs+AFXfD68u9E85cBK5jrTEqyhT1H4BtabE2wzLsphe0ZrtQlDIOZkV/ahbtRAKwevXqg70bBg74diAAW15HbanzORWbcvsNFD8yLphp3dSI/xJgO1998EbaJIR+QrjnH86UdGrE71B+7rYqt7DRGGnLSM0OP/n4yn9k2Ei7EgU7XvR21Dw4bhIs1E4kfvjb0oDRxb2PsfWuL3qH3/BCSMX5ysudnXt6Oe4//+EVvmn56aQ5vGsp59z6Mnv3DwTM8+4vH01u9wchdQu86P5VvLW1E4CHrziWRbNzx5wHhL6/IsH4cQ72mfto+blzuOlvB78xuf3sMi6bV+Rh3y+XHMO3H3+HP11VzbzDEmdK8BU27O5qR82r6upYaEE5CfWdSChE4juR793/ND/bcVnAtEW9fwqpDI1Go4kWoz2M+iOZvxOJCdnbAs0iqtFoNPZFO5EAiBi13SenhPYmo9FoNMmObhMJwIIFC2BwkFqH/+8Zlx/4Ci8PJkfPLI1Goxkr+k0kAE1NTfDidQHTrFfFbFN5FlmkCYWjXRNjbYJGk7RoJxKAsrIy1JqHYm1GzMhIcwCw5LjCGFuiced7i0tibULEOCTLGXYePz6njOtPnR0Ba5KLpltOsqQc7UT8oRQdz96KBPU1QvSYMM4R8rYLZwc1MZlfnvtWLcvPncNtZ5WFlY8vrl10eMTz9Iev7ziuWXRYxPKvOnTymLd54du1PsOH3pqWHDfDK+6RK47l91cexzdOLB5zefFI6bRsli7wPA6LApyzd35pDqeWeb/1X1pTxDUWnk9jZdJ43x85blp+Oq0/OSNq5W7/cEvU8nZHOxF/fPhP8t79daytCIvvnDQrrO0Pz83kq9WHkp7m4Nth5jWS7yyObH4BEW83cv2pR1B7ePjfdiw5bgbfP2XsbwZl031XsQ31uL/42EKuP3U2490eIhbOzmV+iedN9gtHhPY9zljIjcDbgi8e/+bxpJiHZvrEdAAcKf5vSZccf2jI3x+N5LyKqM1l50WgYyQi/PhLc3zGTZ+YzrPXnhByuSPnOYkW2on4Y2B/UMk+Goz+RRwq0ydlRCyvTGfob0S+iIevvH34ljHjmpyBjFCTmhJ+xiJwzaLDef5bvt9Yhji5NLrtcTkTxvHWTYt9xh2RnxVW3qHs/wuqCll+ru+b7lgI1/ZgyUhzeJ0fQwyF5men+4z/140ncbRrUpQsixzaiYRBWe+DbMf/63dWemw7v+X5OTlDITcrcnlpwsP95pvk3wp74UgRvlp9aKzN0LihnUgY7CXwk/75Fa6A8YnEOXOnx9oEWzHkHCQSr0thEM3SA431Fm3iyfnGw1t5OGgn4pfAh9aqYU7i5VyP9M0s1jfHeGWsuyXRd2O8nN/JyMyZMy0pRzsRjSZM1IhboZU39nh6oh4r7g8SdnyoiLbmrCxr2n20E9EkPfa7PUWOaN7n9HGJLp2dnZaUo51ICLQMWtc9UBM+ifSQO/LNYrRRthNJm8Y30XqZzMuzZiQN7UT8EeDqPG3/f1loSHKSLPc+Xzd5f106x0KwziGS+/Gscl+dJ6J3pERiVx1npfONlaMPdQqMsaKdiD8CnN2DerclBdGqkx7ZRhJSHkFmEcl78B1nR35kgnglHtqSZMRvoqLvhmPk5gNXxNoETRwhIhF58wiUv1dYlMpL8VVWgKLCdcKC6Oq4JEA7kSgSiVkj4+GJKRok083Dq3dWGDf5MXfxDbmk+CBZz287EXdORESWich2EVlrLme4xd0oIptFZKOInBplQ3wG96jojCOkiR6JfqMNRCLfg5PpQSIUoq0/NdWaETPidVKqnyml7nEPEJFS4GKgDJgOvCYiJUqpgahYMHDAZ/DfBgOPZaTRhMOYe2dFsmwfLsnm93lLiOSDQH52Ojv29AJQW2vNvSru3kQCcA7whFKqTynVCmwGjotaaQ2P+Ay2ulE9WZ/WrPy4LJplRaLK0hdB986K8m5M3vPPwrKsK4p//GDB8P/GxkZLyozXN5FrReRrwBrgB0qpXUAB8KZbmjYzzAsRWQbcBpCbm0tdXZ1HfH5+PsXFxbS0tFBWVsbKlSu98pi/f1/Y7mIwxm0iI3WHun1qamrEn2qampoiml8gdu/e7RXW0NAQsfzfe+89j/VBNTjqNv70d3V1AYZ9RdnHsn37J8NxQ8ejb+DgSRGJc2wIX205fX376enp8Zm+u7s7rPLeeGMlmz/qB6C/33jz7zb1+6K/v5+WlhaKiz3nUwnlPO/vj04FhjeKHTt2+IxpbGzkmGOO4cMPt/qM/+yzz8a8j99eVT/8Pz8/n507d9Lb20tWVpbXOZeRkUF1dTUNDQ1UVlbS2NjInj17xlQexMiJiMhrQL6PqJuA+4AfY7zl/Rj4f8CVY8lfKbUMWAZQVVWlFi5c6DPdUUcdBYDP+D/+aixF+sRXbxcrWbhwIbz8QnjbR4ny8nJ4vC1q+bszceJE+HyXR1hlZSXS9FZE8j/yyCOhfvXwekpKCgwGdiT+9GdlZcGe3VRUVJKdnY3L5cB46T54PHr2D8CrLwPR66k1RLrTSUaG74FGMzMzoWvsN50h5s+fz7Y3P4L3N5Camgb0k5mVBezzmT41NZXS0lKv8OHzdAznusMR2akN/CNMm5YP272PdWVlJQBFhxYBn3nFT506lalTpwLNQZfmfs13dHQM3+OG43wwZEdFRUXQ5bgTEyeilPI9QcEIROS3wPPm6nbAfZ5WlxkWHTa/FrWsNclDpLv4BvPcES9VTOGaEW0HGC/EqgdaR0eHJeXEXZuIiExzWz0XWG/+fxa4WEScIlIMzAIi8ygZJSJx7kTiwzVNdInlMYplm0i4qkUSu3dZsCS7xnhsE7lbROZi7PutwL8BKKWaReTPwAagH7gmaj2zNEmFv/tgMlzcsfzOIpL+K17erjRjJ+6ciFLq0gBxy4HlFpqj0ViKnQZgTGDTI0qiP8zEnROJV5oHD+WB/rMsL9cu9cbRxN+NNhJ71vcAjOET/ACM0T0/rDr7rH6jSmTnGyyzZs2ypBztRILki/t/EpNydZuIPYmb6XEDlB+uabHUFk/DrURrLzid1oyuEXcN6xqNnUmkJ+RI3ogTSfdYiZW0cL/jCRbtRDQaTUxIYr/hQaxeenJyciwpRzuRKBKJJ7V4eu3WeBPr6qZYEn51VvSGjYknYiXRqmFPtBMJgj6lm44SGX+Nz8lw+0qWNjMb++KERzuRILirf0msTdDEKZF+kh77KL6Je/d1f4uzwQtJ0qKdSBA8NzAvpO0i8XSln9AigMX70NJjpkfx1cQY7USC4DMmhrSdbhPRjJWRN+1Yt7loJxI+sdqHuouvRhNl9P0xfkhmZxWrB8GamhpLytFORGNbonVtJ3I7xUiSSYvdiOScOYHQTiSKRKLnjK7Nim98VTclS4+p0Ujmt4dkYGiekGijnYgm6Umke91Ye2clC3Z+44nWEbZq9lDtROIc+15a0ScSN2jfAzBaOH98tPO3SEoyv73FStuuXbtGTxQBtBOJInZ+utKERtz1zopp6fYg0fexdiJRRLeJxAfJXHcfy/NDPyQFR7LvJ+1ENLYlYk/5SezpA+2jSFbTJPONNpmr6kA7EY2NiVajtZVvPlFvE4ly/proMXv2bEvK0U5Eo4kwydShKpCUZH57SAYcDocl5Wgnokl6on6zS+J7aRJLs44YPVT09vZaUo52IhpNhLG0OivahWkvkrBkZWVZUk7QTkRExkfTEI1Go0lKYuSI4+ZjQxGZJyIbgPfN9XIR+U3ULUsCIlI3nkT167Ei6m8G+hiFTDK1H/klyTUG8ybyM+BUoANAKdUEzA+nUBG5QESaRWRQRKpGxN0oIptFZKOInOoWfpoZtllEbginfI0mmiRTDZBVWpL5W55kJ6jqLKXUthFBA2GWux44D1jpHigipcDFQBlwGvAbEXGIiAP4NXA6UAosMdNqNJoYoW/8GoBgJg/fJiLzACUiacB3gPfCKVQp9R74bBQ8B3hCKdUHtIrIZuA4M26zUmqLud0TZtoN4dih0Wg0yUpGRoYl5QTzJrIUuAYoALYDc831aFAAuL/1tJlh/sJ9IiLLRESJiNq2bRt1dXUey/vvv09fXx/r1q1jcHDQK76urm5UQ1OD2HNLjnWNnmgU/vPcOeRk+O7vnRrgSfDKOeOoq6ujtiCVL82dHlLZQ/uivr4egEMnGs8c6X4KPnvOIT7Dv1CYygmucZxfcXB/+Gr0S3MczDc9mB1sckyu//7wOenC5597D0TX0NDAVSfO9Aib7wrmmcqTs8qn07dj0/B6ugO+ekQqlYXZACz/4mHDcfkTDH3F2Sk0NTVx8bGFRrklB/fbyXm9ZKTCJxvfYc+ePez/vH04buh4rFx58AW+9rAcn3Ydmn1w/01KdzAte/RZ7iY4vffjmYX99PT0UDR5HNcvnsnR0w72r1mc10vNNM9tMn3kAXBkfqZXWF1dHZO6t+IQOKnYyLe7q8uvff39/WzYsIGenh5mTTX0LDli3PB++c4JecNpHSkHz6WCTM/zNTUFTpo9hRkT0zhzZppXOempKUxKP6gj3QGVeb51nTQzkzOKvfMY4pbTZ7G7o90rvHjqBBobGxkcHCRj3ycecU4HnFmaw2effcbWrVs94lxuWq5ddDilUz3LrqurozArhYtmj2PGjBns3LmTjz76iF27dnnd41avXg0cnHeksbEx6PufOxK9r3blNSDfR9RNSqlnzDR1wHVKqTXm+r3Am0qpP5rrDwIvmdudppT6hhl+KVCtlLp2NDuqqqrUmjVrxi5g2cEpcYt6/+QRtfWuLwJQctNL7B8YHA4/Z+50nln7sUeaohteCLrIu88/mvrNn/Fs08deZbV+tpdF99QNh996ZilX1hZ75b/lP88gJcX3TX5k2nGpKezvH/Qop6GhYdR5CBb+9HW2duzj9esWUjx1glf8r1/fzE9f2ci/bvgC0yf5fxr638Y2vv/ng87k4SuO5YqH3x62Z8jeZWeVMn5cKj986l2P7QsmZbD98x7qf7QI1+TxXhqHNF3yu9XUb/5sONyZmsLGO09nT+8Bjl72dwCmZjpZc/Ni3vloF3988yOuP3U2+RPTfdo9VMZQ/uHyaVcfxy5/jSkTxtFwy8mjltd7YIAjbnl5WMcQS//QwMvNO/jnDV+gwM9+H8qrbHo2zR/vGQ5/5poTKC+cFJS933tyLX97Z/uwPRU/fpXOvfs9zvnT5+Rz3yWe59FZv6pn3fbdPHvtCRzt8ixr884uFv/3Su44p4xbn2mmMCeDN374BZ/HMxSCOa/d+ahjH/N/+vqwHQArWj5lyoRxnPmrehaUHMKjVx43nH7omnjjh4sozBnvccxeXv8JS//YyG1nlXL7cxu4+/yjudB8gAiGobzKCyfxzDUneMXf/PQ6/vjmR7z/49NITzvo7Maq2R0RaVBKVY2eMkB1loj8igD9CpRS3w6UsVJqcTAGjGA74L53XWYYAcJjxsgxcaZMGNucxk23nUL57X/3G7/83DnD/4unTqDxlpP55T828ci/tvrdxp8DceedW07mmB+/6jMuEhPZ/PuCw1hy3AxyJowLmO68ChcLSg5h6R8beHvrLr+NuJfNK+Iva9rCtguMm2VJnmf/+RSB+h8tAuCYGZM5ZsbkiJQVLJFqW7jvkgp27Tsw6n4H+N+r5zH75pcBaLh5MVMygz93f/rlo1l2Vtnw+r9u+AIDgwevhabbTmH8OO8n90BjSB2em0XjLSczeXwa51W4SA3iPB4LYz2vfdm6wHxjfHfZKaSneup7/bqFfvf9aXOmDe/js8unB3V83HnsG9V89Xer/cbffvYcrjtltocDgfiYlGoN0ACkAxXAJnOZC4xtLwTPs8DFIuIUkWJgFvAW8DYwS0SKRWQcRuP7s1GyIWQuqBpb9dXEjBGvwW7Xzc8vmstXqw/1iB7ryeePDB8X+BCNjY1h55+SIkHbOiXTSca4wNVIIhKxQezGj3MM6x+6Efz7wsO8LsBERCT4/e5MdXDC4VMAxuRAAFIdKUwcf/DcTU9zMMF58BhOzEgjzTH275hzJoxDRMh0pkb8eETivB4iOz2NcSOqW0fb90P7eEqmc8wfiAa6XsGoups03rvsSGoOhN+rVyn1KICI/DtQq5TqN9fvB94Ip1ARORf4FXAI8IKIrFVKnaqUahaRP2M0mPcD1yilBsxtrgVeARzAQ0qp5nBsiAYpYzg5/FU3DDFaVtHqGbNnz57REyUYs/OzPKqzhhiXmsLDp45n4UJrBqrzR6y+lfjj16stLS+WY22N9byOx3HBxmqRVddyMI8Lk4Fst/VMMyxklFJ/U0q5lFJOpVSeUupUt7jlSqnDlFKzlVIvuYW/qJQqMeOWh1N+pAj14n/pOyfy/LdqAfju4lnD4e4Ny1YQamlXLzocgNyssT3B+iPaU8DecPoRfuNEZMxPhke7JnLirKnhmuXDlrGld3/6D628sWvXaEYSzFl4F/COiLyOcd+ZDyyLplHJzpHTDvrk7y4u4eu1xTywYgtnHT2d19//NOR8v7e4ZNQ0Wenh3XgALqwq5MKq4BsGo0WwzieUqpVAPHttbUTzG400h3Bg4KDW9DQHt5xZyqLZvnvDBeLFb5/Ik6/5r1+PJok0r0Yi2RprRr2jKKUeFpGXgGqMhvYfKaV2RN2yBMTXQ93MqRPY8tnegNtlpadx3alGlcpop26gG+eZ5dMCbvvWf5yEc0Rd88xDJnh1dbWaYJ6Gx4/SbjKSL1f6bp9KxAfvNTedTN+A5/e9X68tDimv0unZLJrhv0uqFcRjVZE/4sHWeB8aJthHtOOAEzHeQo6NnjnJx5+uOt7j+4hJ4yNzAY88tf/r/KM47BDvvvju5GanezXm/98PFrLkuBnD66WlsR0IYOYh3l2GAb541DRuP7uMI/KNnlX3fbUioPO554LyoMqLtd5gmDg+jdws392NQyERNEcarTl6jPp4JyJ3YTiOx8ygb4tIjVLqP6JqWQLw8BXHcumDbwVMkz8xnS8dM52nGtuomDGJP111fMTtOOyQCVx07IzRE5o4U1NYfGQel88r8oobGAh3RJux4/529dJ3TqTfrLopmjKerR37AKPH12Xzirj4uEIODCgynanc+ULwAydMm5jOJ7u951eIhd6R5EwYx6LZh/BvCw4bPXEEiAfNY2VWbuAHpNFIRM1DhPr2bJXmYOoIzgDmKqUGAUTkUeAdwPZO5MRZnnXSSsHfrp7n9b3I4eYF8JXqQ8Puujjy6fuxb1QzO39s8waICL+7zPd3RIODgz7DrUBEcKY6GGov/svSeWxq9/yC2T1+LGSaG42sGoil3iEcKcLDVxw3esIIESvNY+m9OJLHvxnew1eomkM1eVxqCiV54Tm+cLHqOAd7OU4COs3/EwMltCtXnlDMrNxMnx/7TZuYEfTXtiWmw8nL9l198b3FJfQeGBh+8zjh8Mj2EnI6I9PjKhIckuXkkAA9wMbSq+t/vlbFH1Z96FXlF096rSJWmu9dUsFD/2ylbHr26IlHMHWVE2aaAAAgAElEQVSM37KMZKyaZ+SM5xu1xSypDv4N350Wt5EEYoVVxzmYNpGfYPTOesR8C2kA4qKLrRV8NBhcD5hbzyoN6mvx0bh60eH8ZWkNx8+c4jN+4vg07jr/6FE/QAqV9evXRyXfaBJMw3zx1Ak+j1Ei6g2XWGmeMWU8y84uG9N1cmzR5OGPIsNhrJpFhJvPLB21ndEKQm1Yt+o4B9M763FzjKuhBnVb9c7arkZ3Il5fnoeBI0U4tsj3oHrJynFFObyx6TOm+xmraiy8/N0TafjQe8BFTWLyl6XzYm1C3BCvPQuDaVg/AVirlHpWRC4Bfigiv1BKfRh98+KPw0b0HnrrppOSYsiMWHLNosM5s3y6z8Ecx8oR+dkckT/26hKNRhMawVRn3QfsE5Fy4PvAB8Dvo2pVHPPlSs+P7HKz0slOj22/+0QnJUUi4kA0Go31BONE+pXRgnkO8Gul1K+BsXUHSmBGVkcm+5esmZmxrwMOlkgciUTSGym05sRiaJSJQ3PGj5LSE6s0B9M7q0tEbgQuAeaLSAqgH72TlKqqoKYQSBrsphe05kSjJC+LBy+rouawsXUwsEpzMG8iFwF9wNfNBnUX8NOoWhXHxMMwCNFkaJazRCASRyKR9EYKrTnxOOnIvDEP/WOV5mB6Z+0A/ttt/SNs3CYych6BZMOqiWziBbvpBa3ZLsR8UioRqTd/u0Rkz8hfS6yLQy49/tDREyUwIU0lnMDYTS9ozXbBKs2BJqWqNX9t04jui5FVJsn+JtLd3R1rEyzFbnpBa7YLVmkOqpJNRCqAWowOMfVKqXeiapVGEwTJ3U9Oo0kMRn2sFpFbgUeBKcBU4BERuTnahsUL+kYV/yR3VweNJr4J5k3kq0C5UqoXhoeGXwvcGU3DNBqNRhP/BFPB/zHgPqiRE9geHXPij40q9tPAWsmcOXNibYKl2E0vaM12wSrNwbyJ7AaaReRVjNqdk4G3ROSXAEqpb0fRvphzb/+XYm2CpfT19cXaBEuxm17Qmu2CVZqDcSJ/M5ch6qJjSnwyYL6s3XXeUcwY47ADiUhKSuL0Pls4O5fH3/qICaHMUmWSSHojhdZsD6zS7PfqE5FspdQepdSjPuJmmB8d2oaLjwttcppEw+FInBGJ7zinjGu/cHhYQ/Enkt5IoTXbA6s0B3JVdUN/ROQfI+KeDqdQEblARJpFZFBEqtzCi0SkR0TWmsv9bnGVIrJORDaLyC8lmJmIIoCyWd+fDRs2xNqEoElzpFAwKSOsPBJJb6TQmu2BVZoDORH3u+fIWZLCvbOuB84DVvqI+0ApNddclrqF3wdcBcwyl9PCtEGj0Wg0YRLIiSg//32tjwml1HtKqY3BpheRaUC2UupNc1j63wP2avHWaDSaOCRQi2SuiHwf461j6D/menATj4dGsYi8A+wBblZKvQEUAG1uadrMMJ+IyDLgNoDc3Fzq6uo84vPz8ykuLqalpYWysjJWrvR+IVo4Yn1kHgUFBbhcLlpbWykpKaG+vt4rj9raWlpaWiguLqatrY3t2z17RhcWFpKXl0dbWxvFxcWsWrXKK4/58+fT3NxMSUkJra2t7NjhOTNxUVEROTk5tLe343K5WL169ch9wYIFC2hqaqKsrIyWlhZ27tzpkWbmzJlkZWXR2dnJwMCAl9bU1FRqa2tpbGxk7ty5NDc309HR4ZFm1qxZOJ1Ouru7ycnJobGx0SPe6XRSU1NDQ0MDlZWVNDU1sWuX5zS2s2fPxuFw0NvbS1ZWFk1NTR7xGRkZVFdXD+fR2NjInj2ew7iVlpYyMDDA4OAgTqfTa57pzMxMqqqqhvPYt2+fl945c+bQ19dHSkoKDofDq1ogOzubioqK4TxWr15NT0+PR5ry8nK6urpIT09nYGCAjRs9n5kmT55MeXn5cB6rVq3y6k1TUVFBZ2cnmZmZ9PX1sWnTJo/4KVOmUFZWxtq1a6moqKC+vp7+/n6PNFVVVbS3t5OTk0NXVxdbtmyhu7t7WHNubi4lJSU0NzdTXl7OihUrUCMm9a6urqatrY28vDw6OzvZunWrR3ww11NNTQ2tra24XC7a29vZtm2bR3y0r6chzVZfT3l5eV5jWFl1PR04cICdO3eGdT0Fg4w8YYYjRG4LtKFS6vaAGYu8BuT7iLpJKfWMmaYOuE4ptcZcdwKZSqkOEanEaHspA0qAu5RSi810J2LM9X5mIBsAqqqqVEgDkS2bCEB57/+wm0y23vXFseeRgDQ2NlJRURFrMyzDbnpBa7YL4WgWkQalVFATkgQagDGgkxiNoRv+GLfpw5i7BKVUg4h8gOFAtmPMYzKEC4s+eLTbsCd2u9Dsphe0Zrtglea46jwtIoeIiMP8PxOjAX2LUuoTYI+IHG/2yvoa8EwMTU1aEn3ynrFiN72gNdsFqzTHxImIyLki0gbUAC+IyCtm1HzgXRFZC/wVWKqU6jTjrgZ+B2wGPgBesthsW2C3yXvsphe0ZrsQ80mpAEQkRUQujHShSqm/KaVcSimnUipPKXWqGf6UUqrM7N5boZR6zm2bNUqpOUqpw5RS1yp/jTmasBjZkJjs2E0vaM12wSrNAZ2IUmoQ+KEllmjigpG9jJIdu+kFrdkuWKU5mOqs10TkOhEpFJGcoSXqlmk0Go0m7glm5LqLzN9r3MIUMDPy5sQj9hr2RKPRaMbCqE5EKVVshSHxi2560Wg0Gn8EMz3ueBG5WUT+x1yfJSKjfuSXTJxf4Ro9UZJQXl4eaxMsxW56QWu2C1ZpDqZN5GFgPzDPXN+OrabGFQpzwhspNpHo6uqKtQmWYje9oDXbBas0B+NEDlNK3Q0cAFBK7UM3FCQt6enpoydKIuymF7Rmu2CV5mCcyH4RycBsHBCRwzCHJtEkHwMDA7E2wVLsphe0ZrtgleZgnMhtwMtAoYg8BvwD/e1I0jJytNlkx256QWu2C1ZpDqZ31qsi0ggcj1GN9R2l1GdRt0yj0Wg0cU8w34kALABqMaq00oC/Rc0ijUaj0SQMwXTx/Q2wFFiHMa3tv4nIr6NtWLxgtznWNRqNZiwE8ybyBeDIoQEPReRRoDmqVmlixuTJk2NtgqXYTS9ozXbBKs3BNKxvBma4rReaYbagC/t8IwL2+yjLbnpBa7YL8fSxYRbwnojUicjrwAYgW0SeFZFno2tePCCIjaq07DZ5j930gtZsF6zSHEx11q1RtyLOUTYaP8tuk/fYTS9ozXYhLialAlBKrQi0WGGkxjpWrVoVaxMsxW56QWu2C1Zpjqs51jWxp6/PXoMR2E0vaM12wSrN2oloNBqNJmS0E9FoNBpNyPhtWBeRdQSYkUkpdXRULNJoNBpNwhCod9bQxFND0+L+wfz9avTM0cSaioqKWJtgKXbTC1qzXbBKs9/qLKXUh0qpD4GTlVI/VEqtM5cbgFMssU5jOZ2dnbE2wVLsphe0ZrtgleZg2kRERE5wW5kX5HaBMvypiLwvIu+KyN9EZJJb3I0isllENorIqW7hp5lhm0XkhnDK1/gnMzMz1iZYit30gtZsF6zSHIwzuBL4jYhsFZGtwG/MsHB4FZhjtqu0ADcCiEgpcDFQBpxmlusQEQfwa+B0oBRYYqbVRBi7dYW0m17Qmu1CXHTxFZEU4HClVDlQDpQrpeYqpRrDKVQp9XelVL+5+ibgMv+fAzyhlOpTSrVijNF1nLlsVkptUUrtB54w02oizKZNm2JtgqXYTS9ozXbBKs0Bhz1RSg2KyA+BPyuldkfJhiuBJ83/BRhOZYg2Mwxg24jwan8ZisgyjBkZyc3Npa6uziM+Pz+f4uJiWlpaKCsrY+XKlV55LHT7v6tzl1ceBQUFuFwuWltbKSkpob6+3iuP2tpaWlpaKC4upq2tje3bt3vEFxYWkpeXR1tbG8XFxT6/MJ0/fz7Nzc2UlJTQ2trKjh07POKLiorIycmhvb0dl8vF6tWrR+4LFixYQFNTE2VlZbS0tLBz506PNDNnziQrK4vOzk4GBga8tKamplJbW0tjYyNz586lubmZjo4OjzSzZs3C6XTS3d1NTk4OjY2ezxlOp5OamhoaGhqorKykqamJXbt2eaSZPXs2DoeD3t5esrKyaGpq8ojPyMigurp6OI/Gxkb27Nnjkaa0tJSBgQEGBwdxOp2sX7/eIz4zM5OqqqrhPPbt2+eld86cOfT19ZGSkoLD4WDDhg0e8dnZ2VRUVAznsXr1anp6ejzSlJeX09XVRXp6OgMDA16zzE2ePJny8vLhPFatWuX15FhRUUFnZyeZmZn09fV53RSmTJlCWVkZa9eupaKigvr6evr7+z3SVFVV0d7eTk5ODl1dXWzZsoXu7u5hzbm5uZSUlNDc3Ex5eTkrVqzAHLB7mOrqatra2sjLy6Ozs5OtW7d6xAdzPdXU1NDa2orL5aK9vZ1t27Z5xEf7ehrSbPX1lJeXx5o1azzirbqeDhw4wM6dO8O6noJBRp4wXglE7gI+w7jR7x0KV0oFbLURkdeAfB9RNymlnjHT3ARUAecppZSI3Au8qZT6oxn/IPCSud1pSqlvmOGXAtVKqWtHE1hVVaVGHsSgWDYRgKLeP/HdxbP47uKSseeRgNTV1bFw4cJYm2EZdtMLWrNdCEeziDQopaqCSRvMAIwXmb/XuIUpYGagjZRSiwPFi8jlGN2IT1IHPdl2jKHmh3CZYQQIjzqLj8yzqiiNRqNJKIKZY7040oWKyGnAD4EFSql9blHPAn8Skf8GpgOzgLcw5nafJSLFGM7jYuArkbbLF99bXMKcgolWFBUXTJkyJdYmWIrd9ILWbBes0hzUHOsiMgejV1T6UJhS6vdhlHsv4AReFREwqrCWKqWaReTPGHOW9APXKKUGTBuuBV4BHMBDSilLZlf8twUBX7iSjrKyslibYCl20wtas12wSnMwc6zfBvzKXBYBdwNnh1OoUupwpVSh2dNrrlJqqVvccqXUYUqp2Uqpl9zCX1RKlZhxy8MpfyyIfeajAmDt2rWxNsFS7KYXtGa7YJXmYL4T+TJwErBDKXUFRldf+9Tv2Ay7DQ9hN72gNduFmA974kaPUmoQ6BeRbGAnno3cSY2dpsYFfHatTGbsphe0ZrtgleZg2kTWmMOS/BZoALoB+00TZhNGfmeQ7NhNL2jNdsEqzcH0zrra/Hu/iLwMZCul3o2uWRqNRqNJBEZ1IiLyB2Al8IZS6v3om6TRaDSaRCGYNpGHgGnAr0Rki4g8JSLfibJdGo1Go0kAgqnOel1EVgLHYnTxXYoxyu4vomxbXGC3Lr5VVUGNdJA02E0vaM12wSrNwXwn8g/gnxjDn2wEjlVKHRFtwzSxob29PdYmWIrd9ILWbBes0hxMdda7wH5gDnA0MEdEMqJqlSZm5OTkxNoES7GbXtCa7YJVmkd1Ikqp7yml5gPnAR3Aw8Dn0TZMExu6urpibYKl2E0vaM12wSrNwVRnXSsiTwLvYEwE9RDGDIOaJGTLli2xNsFS7KYXtGa7YJXmYD42TAf+G2hwm43QNtisXV2j0WjGRDDVWfcAacClACJyiDkku0aj0WhsTrCj+P4IuNEMSgP+GE2jNBqNRpMYBNM761yMod/3AiilPgayommUJnbk5ubG2gRLsZte0JrtglWag3Ei+83paxWAiEyIrkmaWFJSYo+55Iewm17Qmu2CVZqDcSJ/FpEHgEkichXwGvC76JoVP4jNPllvbrZkwsi4wW56QWu2C1ZpFuMlY5REIicDp2B0VnpFKfVqtA2LFFVVVWrNmjVj33CZMe/WwK2f40ixlyPRaDT2RkQalFJBjZsSzJsISqlXlVLXK6WuA/4hIl8Ny0JN3LJixYpYm2ApdtMLWrNdsEqzXyciItkicqOI3Csip4jBtcAW4EJLrNNYTjBvpsmE3fSC1mwXrNIc6GPDPwC7MGYx/AbwHxjVWV9SStlm1ntdkaXRaDT+CeREZiqljgIQkd8BnwAzlFK9llim0Wg0mrgnUJvIgaE/SqkBoC1SDkREfioi74vIuyLyN3MOd0SkSER6RGStudzvtk2liKwTkc0i8kuxW7cpjUajiUMCOZFyEdljLl3A0UP/RWRPmOW+CsxRSh0NtHDwa3iAD5RSc81lqVv4fcBVwCxzOS1MGzQ+qK6ujrUJlmI3vaA12wWrNPt1Ikoph1Iq21yylFKpbv+zwylUKfV3t8Ec3wRcgdKLyDQgWyn1pvnh4++BL4Vjw2jM6/0lJ/TaYvJGD9ra2mJtgqXYTS9ozXbBKs1BdfGNMlcCL7mtF4vIOyKyQkRONMMKAPc90maGRY2Pmcp2DrHd9Lh5eXmxNsFS7KYXtGa7YJXmYIaCDwkReQ3I9xF1k1LqGTPNTUA/8JgZN9R43yEilcDTIlIWQtnLgNvAGD+mrq7OIz4/P5/i4mJaWlooKytj5cqVfvPq7u5m586dbNu2zSO8oKAAl8tFa2srJSUl1NfXe21bW1tLS0sLxcXFtLW1sX37do/4wsJC8vLyaGtro7i4mFWrVnnlMX/+fJqbmykpKaG1tZUdO3Z4xBcVFZGTk0N7ezsul4vVq1eP3BcsWLCApqYmysrKaGlpYefOnR5pZs6cSVZWFp2dnfT399PY2OgRn5qaSm1tLY2NjcydO5fm5mY6Ojo80syaNQun00l3dzc5OTleeTidTmpqamhoaKCyspKmpiZ27drlkWb27Nk4HA56e3vJysqiqanJIz4jI4Pq6urhPBobG9mzx7NmtbS0lIGBAQYHB3E6naxfv94jPjMzk6qqquE83nnnHa+ukHPmzKGvr4+UlBQcDgcbNmzwiM/OzqaiomI4j9WrV9PT0+ORpry8nK6uLtLT0xkYGGDjxo0e8ZMnT6a8vHw4j1WrVtHX1+eRpqKigs7OTjIzM+nr62PTpk0e8VOmTKGsrIy1a9dSUVFBfX09/f2eszVUVVXR3t5OTk4OXV1dbNmyhf379zNu3DjAuD5KSkpobm6mvLycFStWeO2P6upq2trayMvLo7Ozk61bt3rEB3M91dTU0Nraisvlor293fLraUiz1ddTXl4eIz92tup6mjx5MtOmTQvregoKpVRMFuByjO7D4wOkqQOqgGnA+27hS4AHgimnsrJShcKhP3peHfqj59Xg4GBI2ycqr7/+eqxNsBS76VVKa7YL4WgG1qgg7+Uxqc4SkdOAHwJnK6X2uYUfIiIO8/9MjAb0LUqpT4A9InK82Svra8AzMTBdo9FoNG5ErTprFO4FnMCrZk/dN5XRE2s+cIeIHAAGgaVKqU5zm6uBR4AMjDaUl0ZmqtFoNBpriYkTUUod7if8KeApP3FrgDnRtMsX+nMUjUaj8U889M7SxBH5+b76QiQvdtMLWrNdsEqzdiIaD4qLi2NtgqXYTS9ozXbBKs3aiWg8aGlpibUJlmI3vaA12wWrNGsnovGgrGzMn+UkNHbTC1qzXbBKs3YiGg8CfXiZjNhNL2jNdsEqzdqJaDQajSZktBPRaDQaTchoJ6LRaDSakNFORKPRaDQho52IxoOamppYm2ApdtMLWrNdsEqzdiIaD1pbW2NtgqXYTS9ozXbBKs3aiWg8cLkCTjKZdNhNL2jNdsEqzdqJaDxob2+PtQmWYje9oDXbBas0x2ooeE2csm3bNg477LBYm2EZdtMLiav5wIEDtLW10dvbO+Zte3t7ee+996JgVfwSjOb09HRcLhdpaWkhl6OdiEajSQja2trIysqiqKhozFM0dHV1kZWVFSXL4pPRNCul6OjoGJ5OOFR0dZZGo0kIent7mTJlip7jJ0KICFOmTAnpzc4d7UQ0Gk3CoB1IZInE/tRORONBQUFBrE2wFLvpBXtqDqfOP1GxSrN2IhoP7NYV0m56wZ6ax40bF2sTLMcqzdqJaDyw20dZdtML9tTc19cXs7L7+/tjUq5VmnXvLI0HJSUlsTbBUuymF5JD8+3PNbPh4z0RzbN0eja3neV/Iqe9e/dy4YUX0tbWxsDAALfccgsbN27kueeeo6enh3nz5vHAAw8gIixcuJC5c+dSX1/PkiVLmDFjBrfffjsOh4OJEyeycuVKtm7dyqWXXsrevXsBuPfee5k3b17E9KSnp0csr0BoJ6LxoL6+noULF8baDMuwm16wp+aBgQEcDkdYebz88stMnz6dF154AYDdu3dz8sknc+uttwJw6aWX8vzzz3PWWWcBsH//ftasWQPAUUcdxSuvvEJBQQGff/45ALm5ubz66qukp6ezadMmlixZMpw+EnR3d1vSrVk7EY1Gk3AEemPwRSS+EznqqKP4wQ9+wI9+9CPOPPNMTjzxRJ566inuvvtu9u3bR2dnJ2VlZcNO5KKLLhre9oQTTuDyyy/nwgsv5LzzzgOMjyevvfZa1q5di8PhSNh54GPWJiIiPxaRd0VkrYj8XUSmm+EiIr8Ukc1mfIXbNpeJyCZzuSxWtms0GvtRUlJCY2MjRx11FDfffDN33HEHV199NX/9619Zt24dV111lcc3FxMmTBj+f//993PnnXeybds2Kisr6ejo4Gc/+xl5eXk0NTWxZs0a9u/fHwtZYRPLhvWfKqWOVkrNBZ4HbjXDTwdmmcs3gfsARCQHuA2oBo4DbhORyZZbrdFobMnHH3/M+PHjueSSS7j++utpbGwEYOrUqXR3d/PXv/7V77YffPAB1dXV3HHHHRxyyCFs27aN3bt3M23aNFJSUvjDH/7AwMCAVVIiSsyqs5RS7q1iEwBl/j8H+L1SSgFvisgkEZkGLAReVUp1AojIq8BpwOPWWa3RaOzKunXruP7660lJSSEtLY377ruPp59+mjlz5pCfn8+xxx7rd9vrr7+eTZs2oZTipJNOory8nKuvvprzzz+f3//+95x22mkeby6JhBj36hgVLrIc+BqwG1iklPpURJ4H7lJK1Ztp/gH8CMOJpCul7jTDbwF6lFL3+Mh3GcZbC7m5uTz55JMe8fn5+RQXF9PS0kJZWRkrV670su3yl40eE+tvXkB7ezvbtm3ziC8oKMDlctHa2kpJSQn19fVeedTW1tLS0kJxcTFtbW1s377dI76wsJC8vLzhsWtWrVrllcf8+fNpbm6mpKSE1tZWduzY4RFfVFRETk4O7e3tuFwuVq9ePXJfsGDBApqamigrK6OlpYWdO3d6pJk5cyZZWVl0dnYydepU3nnnHY/41NRUamtraWxsZO7cuTQ3N9PR0eGRZtasWTidTrq7u8nJyRl+ShvC6XRSU1NDQ0MDlZWVNDU1sWvXLo80s2fPxuFw0NvbS1ZWFk1NTR7xGRkZVFdXD+fR2NjInj2ePXRKS0sZGBhgcHAQp9PJ+vXrPeIzMzOpqqoazuOtt95i3759HmnmzJlDX18fKSkpOBwONmzY4BGfnZ1NRUXFcB6rV6+mp6fHI015eTldXV2kp6czMDDAxo0bPeInT55MeXn5cB6rVq3y6pJZUVFBZ2cnmZmZ9PX1sWnTJo/4KVOmUFZWxtq1a6moqKC+vt6rO2lVVRXt7e3k5OTQ1dXFli1bPOJzc3MpKSmhubmZ8vJyVqxYwch7QnV1NW1tbeTl5dHZ2cnWrVs94oO5nmpqamhtbcXlcoV8PeXm5lJcXIzT6WT//v0cOHDAIz4tLY20tDT279+P0+kc7vXkTmZmJj09PaSnp9PX1+e1v8aNG0dqaioHDhxg3LhxPvPIyspi3759ZGRk0Nvb6zMPh8NBf38/aWlpXufXUB579+5l/Pjx9PT0eL2FOJ1ORITBwUFSU1O98hARMjMz2bt3LxMmTGDfvn0B83A4HF7n6FAeTU1NlJeXe1xPixYtalBKVXkZ7oOoOhEReQ3I9xF1k1LqGbd0N2I4iNsi4UTcqaqqUqH0eCi6weiBsfWuL45520Rmw4YNlJaWxtoMy7CbXkhcze+99x5HHnlkSNv29PSQkZERYYvim2A1+9qvIhK0E4lqdZZSanGQSR8DXsR4e9gOFLrFucyw7RiOxD28LmwjNR6EM5pnImI3vWBPzU6nM9YmWI5VmmPZO2uW2+o5wPvm/2eBr5m9tI4HdiulPgFeAU4Rkclmg/opZpgmgrS1tcXaBEuxm16wp+ZE7fkUDlZpjmXvrLtEZL2IvIvhEL5jhr8IbAE2A78FrgYwG9R/DLxtLncMNbJrIsfIdptkx256wZ6aR7af2AGrNMeyd9b5fsIVcI2fuIeAh6Jpl0aj0WiCRw/AqNFoNJqQ0U5Eo9FoIsiyZcu4556AnUYjwhlnnDE8Dlcs0WNnaTwoLCwcPVESYTe9YE/NiTgpVX9/P6mp/m/RL774YsDtrdKsnYjGg7y8vFibYCl20wtJovmlG2DHuqCTO1HAKFPB5h8Fp98VMMny5ct59NFHyc3NpbCwkMrKSq677jq/6T/44AOuueYaPv30U8aPH89vf/tbjjjiCJ577jnuvPNO9u/fz5QpU3jsscfIy8tj2bJlfPDBB2zZsoUZM2Zw6qmn8uyzz7Jv3z4++OADzj33XO6++27A+NB4zZo1dHd3c/rpp1NbW8u//vUvCgoKeOaZZxg3bhxvv/02X//610lJSeHkk0/mpZde8voAN1x0dZbGA7t1/7SbXrCn5sHBwbDzaGho4IknnmDt2rW8+OKLvP3226Nu881vfpNf/epXNDQ0cM8993D11VcDxmgWb775Ju+88w4XX3zxsGMA42PQ1157jccfN0Z0Wrt2LU8++STr1q3jySef9PraH2DTpk1cc801NDc3M2nSJJ566in279/PFVdcwQMPPDA8UnA00G8iGg/s9iGa3fRCkmge5Y1hJDI4CCnhPTO/8cYbnHvuuYwfPx6As88+O2D67u5u/vWvf3HBBRcMhw0NbdPW1sZFF13EJ598wv79+z2Oydlnn+3xpflJJ53ExIkTAWNYnw8//NCrSrK4uJi5c+cCUFlZydatW+np6aGrq4uamhoAvvKVr/D888+HKt8v+k1E44Gv8buSGfS3+p4AAAoDSURBVLvpBXtq9jUGVrQZHBxk0qRJrF27dnh57733APjWt77Ftddey7p163jggQf8DiEPnl+eD43JNRJfaazSrJ2IRqPRBMH8+fN5+umnh5/wn3vuuYDps7OzKS4u5i9/+QsASqnhQUV3795NQUEBAI8++mhU7J00aRJZWVnDg7I+8cQTUSlHOxGNRqMJgoqKCi666CLKy8s5/fTTAw79PsRjjz3Ggw8+SHl5OWVlZTzzjDHu7LJly7jggguorKxk6tSpUbP5wQcf5KqrrmLu3Lns3bt3uFosksR0KHgrCHUU3w0f7+Gxv69m+eUnR8Gq+KWurs5W82/bTS8kruZwRvGNxPS4I1m2bBmZmZkBe2fFkq6uruHh3gHuuusuPvnkE37xi194pIvrUXwTmdLp2ZxclHh9yzUajWaIF154gZ/85Cf09/dz6KGH8sgjj0S8DP0mEoDBwUFSwuzRkWjYTbPd9ELiag7nTUQphcgo34kkGcFqDvdNJPHOJAtpbm6OtQmWYzfNdtMLia051IfekbP62YFgNEfiJUI7kQCUlJTE2gTLsZtmu+mFxNWcnp5OR0dHSDe+9PT0KFgU34ymWSlFR0dH2PtGt4kEoLW1lSOOOCLWZliK3TTbTS8krmaXy0VbWxuffvrpmLc9cOBAQo6fFQ7BaE5PT8flcoVVjnYiAdixY0dCXmzhYDfNdtMLias5LS0t5K/tE7VHWjhYpVlXZ2k0Go0mZLQT0Wg0Gk3IaCei0Wg0mpBJ+u9ERORT4MMQN58OfBxBcxIBu2m2m17Qmu1COJoPVUodEkzCpHci4SAiSillqy+U7KbZbnpBa7YLVmnW1VkajUajCRntRDQajUYTMtqJBOb2WBsQA+ym2W56QWu2C5Zo1m0iGo1GowkZ/Sai0Wg0mpDRTkSj0Wg0IaOdiA9E5DQR2Sgim0XkhljbEw4iUigir4vIBhFpFpHvmOE5IvKqiGwyfyeb4SIivzS1vysiFW55XWam3yQil8VKUzCIiENE3hGR5831YhFZbep6UkTGmeFOc32zGV/klseNZvhGETk1NkqCR0QmichfReR9EXlPRGqS+TiLyPfMc3q9iDwuIunJeJxF5CER2Ski693CInZcRaRSRNaZ2/xSxjrxilJKL24L4AA+AGYC44AmoDTWdoWhZxpQYf7PAlqAUuBu4AYz/Abgv8z/ZwAvAQIcD6w2w3OALebvZPP/5FjrC6D7+8CfgOfN9T8DF5v/7wf+3fx/NXC/+f9i4Enzf6l57J1AsXlOOGKtaxTNjwLfMP+PAyYl63EGCoBWIMPt+F6ejMcZmA9UAOvdwiJ2XIG3zLRibnv6mOyL9Q6KtwWoAV5xW78RuDHWdkVQ3zPAycBGYJoZNg3YaP5/AFjiln6jGb8EeMAt3CNdPC2AC/gH8AXgefPi+AxIHXmMgVeAGvN/qplORh5393TxuAATzZuqjAhPyuNsOpFt5k0x1TzOpybrcQaKRjiRiBxXM+59t3CPdMEsujrLm6GTc4g2MyzhMV/hjwFWA3lKqU/MqB1Anvnfn/5E2i8/B34IDJrrU4DPlVL95rq77cO6zPjdZvpE0gvGU/SnwMNmNd7vRGQCSXqclVLbgXuAj4BPMI5bA8l/nIeI1HEtMP+PDA8a7URsgohkAk8B31VK7XGPU8YjSFL09RaRM4GdSqmGWNtiMakYVR73KaWOAfZiVHMMk2THeTJwDobznA5MAE6LqVExItbHVTsRb7YDhW7rLjMsYRGRNAwH8phS6n/N4HYRmWbGTwN2muH+9CfKfjkBOFtEtgJPYFRp/QKYJCJDk7C52z6sy4yfCHSQOHqHaAPalFKrzfW/YjiVZD3Oi4FWpdSnSqkDwP9iHPtkP85DROq4bjf/jwwPGu1EvHkbmGX28hiH0Qj3bIxtChmzp8WDwHtKqf92i3oWGOqhcRlGW8lQ+NfMXh7HA7vN1+ZXgFNEZLL5FHiKGRZXKKVuVEq5lFJFGMfu/5RSXwVeB75sJhupd2g/fNlMr8zwi81ePcXALIwGyLhEKbUD2CYis82gk4ANJOlxxqjGOl5Expvn+JDepD7ObkTkuJpxe0TkeHM/fs0tr+CIdYNRPC4YPRxaMHpq3BRre8LUUovxqvsusNZczsCoD/4HsAl4Dcgx0wvwa1P7OqDKLa8rgc3mckWstQWhfSEHe2fNxLg5bAb+AjjN8HRzfbMZP9Nt+5vM/bCRMfZYiZHeucAa81g/jdELJ2mPM8awHu8D64E/YPSwSrrjDDyO0e5zAOON8+uRPK5AlbkPPwDuZUTnjNEWPeyJRqPRaEJGV2dpNBqNJmS0E9FoNBpNyGgnotFoNJqQ0U5Eo9FoNCGjnYhGo9FoQkY7EY3tEZEBEVnrtgQcuVlElorI1yJQ7lYRmRqBfL47mj0icpSIPBJuWRrNSHQXX43tEZFupVRmDMrditGP/7Mw8kgFGjFGau4fJe1rwJVKqY9CLU+jGYl+E9Fo/GC+KdxtzrXwlogcboYvE5HrzP/fFmOulndF5AkzLEdEnjbD3hSRo83wKSLydzHmwPgdxodhQ2VdYpaxVkQeEGM+FIeIPCLGfBnrROR7Psz8AtA45EBEpE5E/svMq0VETnRL+xzGV/waTcTQTkSjgYwR1VkXucXtVkodhfEl7899bHsDcIxS6mhgqRn2/9u7e9YoojCK4/8nBlQQRQtBxAgiiigYFGxiQLESOxWC+hFsrDRIwOQTCNqKwaCkChEkKCl8RwsVIaAp/AYixMogEo7FcxfHZbOJwyAo5wfL7N65w+w2+8ydvXvuGPC+tF0FJkr7NeClpP3ANNAHEBH7gCFgQFI/sARcIP+Bvl3SgfIexjucf4BMr63qlXQEuFTO2fIWGMSsQb0rdzH77y2WL+9OJivb6x32zwH3IuI+GTUCGTVzBkDS4zIC2UguLnS6tM9ExELpfwI4DLwpi8qtJwP1HgC7IuImMAPMdjj/NmC+ra0VsvmOXIei5TOZeGvWGI9EzLrTMs9bTpFZRYfIIlDnwiyAO5L6y2OvpFFJC8BB4Ck5yrnV4dhFMheq6nvZLvH7heK60t+sMS4iZt0NVbavqzsiogfYIekJcIWMF98AvCBvRxERx4AvyjVcngPnS/tJMiARMkjvbERsLfu2RMTOMnOrR9IUMEIWqnbzwO5VfpY9ZNCeWWN8O8us/CZSef1IUmua7+aImCOv7s+1HbcGuBsRm8jRxA1JXyNiFLhdjvvGr8juMWAyIj4Ar8g4cyR9jIgRYLYUph/ARXLUMF7aIJdybfeQTLBdjePkbTGzxniKr9kympiC+zdExDRwWdKnLn3WAs+AoytNBTb7E76dZfbvGyZ/YO+mDxh2AbGmeSRiZma1eSRiZma1uYiYmVltLiJmZlabi4iZmdXmImJmZrX9BGLEK0X677WuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n = 4 # 去掉早期n个奖励过低的Episode\n",
    "data = sarsa_sta[2][n:], sarsa_sta[1][n:], q_sta[1][n:] \n",
    "learning_curve(data, x_index = 0, y1_index = 1, y2_index = 2,\n",
    "               title=\"compare of Q and Sarsa\", x_name = \"Episodes (n)\", y_name = \"Reward per Episode\",\n",
    "               y1_legend = \"sarsa\", y2_legend = \"q learning\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 10000   17 steps,total reward:-16.00  \n",
      "episode 10000   13 steps,total reward:-12.00  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(13, -12)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sarsa_agent.learning_method(display=True)\n",
    "q_agent.learning_method(display=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "detail of (episode 10000   17 steps,total reward:-16.00  ):\n",
      "step0     s:0   a:2   r:-1   is_end:0     s1:12 \n",
      "step1     s:12  a:2   r:-1   is_end:0     s1:24 \n",
      "step2     s:24  a:2   r:-1   is_end:0     s1:36 \n",
      "step3     s:36  a:1   r:-1   is_end:0     s1:37 \n",
      "step4     s:37  a:1   r:-1   is_end:0     s1:38 \n",
      "step5     s:38  a:1   r:-1   is_end:0     s1:39 \n",
      "step6     s:39  a:1   r:-1   is_end:0     s1:40 \n",
      "step7     s:40  a:1   r:-1   is_end:0     s1:41 \n",
      "step8     s:41  a:1   r:-1   is_end:0     s1:42 \n",
      "step9     s:42  a:1   r:-1   is_end:0     s1:43 \n",
      "step10    s:43  a:1   r:-1   is_end:0     s1:44 \n",
      "step11    s:44  a:1   r:-1   is_end:0     s1:45 \n",
      "step12    s:45  a:1   r:-1   is_end:0     s1:46 \n",
      "step13    s:46  a:1   r:-1   is_end:0     s1:47 \n",
      "step14    s:47  a:3   r:-1   is_end:0     s1:35 \n",
      "step15    s:35  a:3   r:-1   is_end:0     s1:23 \n",
      "step16    s:23  a:3   r:0    is_end:1     s1:11 \n",
      "detail of (episode 10000   13 steps,total reward:-12.00  ):\n",
      "step0     s:0   a:2   r:-1   is_end:0     s1:12 \n",
      "step1     s:12  a:1   r:-1   is_end:0     s1:13 \n",
      "step2     s:13  a:1   r:-1   is_end:0     s1:14 \n",
      "step3     s:14  a:1   r:-1   is_end:0     s1:15 \n",
      "step4     s:15  a:1   r:-1   is_end:0     s1:16 \n",
      "step5     s:16  a:1   r:-1   is_end:0     s1:17 \n",
      "step6     s:17  a:1   r:-1   is_end:0     s1:18 \n",
      "step7     s:18  a:1   r:-1   is_end:0     s1:19 \n",
      "step8     s:19  a:1   r:-1   is_end:0     s1:20 \n",
      "step9     s:20  a:1   r:-1   is_end:0     s1:21 \n",
      "step10    s:21  a:1   r:-1   is_end:0     s1:22 \n",
      "step11    s:22  a:1   r:-1   is_end:0     s1:23 \n",
      "step12    s:23  a:3   r:0    is_end:1     s1:11 \n"
     ]
    }
   ],
   "source": [
    "env.close()\n",
    "sarsa_agent.last_episode_detail()\n",
    "q_agent.last_episode_detail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "def xy2state(x, y):\n",
    "    return str(y*12+x)\n",
    "\n",
    "def str_key(s, a):\n",
    "    '''根据横坐标，纵坐标和行为生成键\n",
    "    '''\n",
    "    return str(s)+\"_\"+str(a)\n",
    "    \n",
    "def print_q(agent):\n",
    "    '''打印输出agent的价值\n",
    "    '''\n",
    "    for y in range(4):\n",
    "        for x in range(12):\n",
    "            for a in range(4):\n",
    "                key = str_key(xy2state(x,y),a)\n",
    "                print(\"{}_{}_{}:{}\".format(x,y,a,agent.Q.get(key,0)))\n",
    "                \n",
    "def show_q(agent):\n",
    "    '''绘制agent学习得到的Q值，以图片的形式，每一个位置用3*3的小方格表示，\n",
    "    中间小方格表示该状态的价值，左右上下四个小方格分别表示相应行为的价值，\n",
    "    四个角上的数据暂时没有意义。\n",
    "    '''\n",
    "    V = np.zeros((4*3,12*3))\n",
    "    for y in range(4):\n",
    "        for x in range(12):\n",
    "            max_qsa = -float('inf')\n",
    "            for a in range(4): # 0-3 分别为 左 右 上 下\n",
    "                key = str_key(xy2state(x,y),a)\n",
    "                qsa = agent.Q.get(key,0)\n",
    "                if a == 0: V[3*y+1, 3*x+1-1] = qsa\n",
    "                if a == 1: V[3*y+1, 3*x+1+1] = qsa\n",
    "                if a == 2: V[3*y+1+1, 3*x+1] = qsa\n",
    "                if a == 3: V[3*y+1-1, 3*x+1] = qsa\n",
    "                if qsa > max_qsa: max_qsa = qsa\n",
    "            V[3*y+1, 3*x+1] = max_qsa\n",
    "    plt.imshow(V, cmap=plt.cm.gray, interpolation=None, origin=\"lower\", extent=[0, 12, 0, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "show_q(sarsa_agent)\n",
    "def value_of(agent, s):\n",
    "    '''以最大行为价值作为某一状态的价值\n",
    "    '''\n",
    "    A, Q = agent.A, agent.Q\n",
    "    max_q = -float('inf')\n",
    "    for a in A:\n",
    "        cur_q = Q.get(str_key(s,a),0)\n",
    "        if cur_q > max_q:\n",
    "            max_q = cur_q\n",
    "    return max_q\n",
    "# 比较坐标(0,0)和(0,3)两处位置的价值大小，判断sarsa学习得到的两处是左上方的价值较大\n",
    "def print_v(agent, x, y):\n",
    "    print(\"{}:value of ({},{}):{}\".format(agent.__class__.__name__,x,y,value_of(agent,xy2state(x,y))))\n",
    "\n",
    "print_v(sarsa_agent, 1,1) # 在(0,1)处右转价值\n",
    "print_v(sarsa_agent, 0,2) # 在(0,1)处向上价值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sarsa 学习在 $ \\epsilon $ - greedy 策略下 $\\epsilon$ 不衰减 时学习到的路线是从上方远离悬崖的地方行走\n",
    "\n",
    "$$J = -\\sum_{3}^{2}{ad} + \\frac{2}{0} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_q(q_agent)\n",
    "print_v(q_agent, 1,1) # 在(0,1)处右转价值\n",
    "print_v(q_agent, 0,2) # 在(0,1)处向上价值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
